fetcher queue: also throttle on outstanding writes

When fetching over a fast enough connection, we can be receiving files
faster than we write them. This can then lead to EMFILE when we have
enough files open. This was made very easy to notice with the upcoming
libcurl backend, which makes use of pipelining.

Closes: #675
Approved by: cgwalters

diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 73e024467025443299e374835e70a6c71f94cbc8..f1e00f27b86657fae4708f013569f8627e637328 100644 (file)
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -35,6 +35,14 @@ G_BEGIN_DECLS
 #define _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS 8
 #define _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS 2
 
+/* In most cases, writing to disk should be much faster than
+ * fetching from the network, so we shouldn't actually hit
+ * this. But if using pipelining and e.g. pulling over LAN
+ * (or writing to slow media), we can have a runaway
+ * situation towards EMFILE.
+ * */
+#define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 16
+
 typedef enum {
   OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0)
 } OstreeRepoTestErrorFlags;

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 295973ec3c488b718eb1e855c4cf2c71992c9c66..0e5128d9ee978e4cb2d64c1e34c6e0ad0f65fb43 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -345,15 +345,26 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
 
 /* We have a total-request limit, as well has a hardcoded max of 2 for delta
  * parts. The logic for the delta one is that processing them is expensive, and
- * doing multiple simultaneously could risk space/memory on smaller devices.
+ * doing multiple simultaneously could risk space/memory on smaller devices. We
+ * also throttle on outstanding writes in case fetches are faster.
  */
 static gboolean
 fetcher_queue_is_full (OtPullData *pull_data)
 {
-  return (pull_data->n_outstanding_metadata_fetches +
-          pull_data->n_outstanding_content_fetches +
-          pull_data->n_outstanding_deltapart_fetches) == _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS ||
-    pull_data->n_outstanding_deltapart_fetches == _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS;
+  const gboolean fetch_full =
+      ((pull_data->n_outstanding_metadata_fetches +
+        pull_data->n_outstanding_content_fetches +
+        pull_data->n_outstanding_deltapart_fetches) ==
+         _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS);
+  const gboolean deltas_full =
+      (pull_data->n_outstanding_deltapart_fetches ==
+        _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
+  const gboolean writes_full =
+      ((pull_data->n_outstanding_metadata_write_requests +
+        pull_data->n_outstanding_content_write_requests +
+        pull_data->n_outstanding_deltapart_write_requests) >=
+         _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS);
+  return fetch_full || deltas_full || writes_full;
 }
 
 static gboolean